{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Varying regularization in Multi-layer Perceptron\n\n\nA comparison of different values for regularization parameter 'alpha' on\nsynthetic datasets. The plot shows that different alphas yield different\ndecision functions.\n\nAlpha is a parameter for regularization term, aka penalty term, that combats\noverfitting by constraining the size of the weights. Increasing alpha may fix\nhigh variance (a sign of overfitting) by encouraging smaller weights, resulting\nin a decision boundary plot that appears with lesser curvatures.\nSimilarly, decreasing alpha may fix high bias (a sign of underfitting) by\nencouraging larger weights, potentially resulting in a more complicated\ndecision boundary.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "print(__doc__)\n\n\n# Author: Issam H. Laradji\n# License: BSD 3 clause\n\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom matplotlib.colors import ListedColormap\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.datasets import make_moons, make_circles, make_classification\nfrom sklearn.neural_network import MLPClassifier\n\nh = .02  # step size in the mesh\n\nalphas = np.logspace(-5, 3, 5)\nnames = ['alpha ' + str(i) for i in alphas]\n\nclassifiers = []\nfor i in alphas:\n    classifiers.append(MLPClassifier(solver='lbfgs', alpha=i, random_state=1,\n                                     hidden_layer_sizes=[100, 100]))\n\nX, y = make_classification(n_features=2, n_redundant=0, n_informative=2,\n                           random_state=0, n_clusters_per_class=1)\nrng = np.random.RandomState(2)\nX += 2 * rng.uniform(size=X.shape)\nlinearly_separable = (X, y)\n\ndatasets = [make_moons(noise=0.3, random_state=0),\n            make_circles(noise=0.2, factor=0.5, random_state=1),\n            linearly_separable]\n\nfigure = plt.figure(figsize=(17, 9))\ni = 1\n# iterate over datasets\nfor X, y in datasets:\n    # preprocess dataset, split into training and test part\n    X = StandardScaler().fit_transform(X)\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4)\n\n    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n                         np.arange(y_min, y_max, h))\n\n    # just plot the dataset first\n    cm = plt.cm.RdBu\n    cm_bright = ListedColormap(['#FF0000', '#0000FF'])\n    ax = plt.subplot(len(datasets), len(classifiers) + 1, i)\n    # Plot the training points\n    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)\n    # and testing points\n    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6)\n    ax.set_xlim(xx.min(), xx.max())\n    ax.set_ylim(yy.min(), yy.max())\n    ax.set_xticks(())\n    ax.set_yticks(())\n    i += 1\n\n    # iterate over classifiers\n    for name, clf in zip(names, classifiers):\n        ax = plt.subplot(len(datasets), len(classifiers) + 1, i)\n        clf.fit(X_train, y_train)\n        score = clf.score(X_test, y_test)\n\n        # Plot the decision boundary. For that, we will assign a color to each\n        # point in the mesh [x_min, x_max]x[y_min, y_max].\n        if hasattr(clf, \"decision_function\"):\n            Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])\n        else:\n            Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]\n\n        # Put the result into a color plot\n        Z = Z.reshape(xx.shape)\n        ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)\n\n        # Plot also the training points\n        ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,\n                   edgecolors='black', s=25)\n        # and testing points\n        ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,\n                   alpha=0.6, edgecolors='black', s=25)\n\n        ax.set_xlim(xx.min(), xx.max())\n        ax.set_ylim(yy.min(), yy.max())\n        ax.set_xticks(())\n        ax.set_yticks(())\n        ax.set_title(name)\n        ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),\n                size=15, horizontalalignment='right')\n        i += 1\n\nfigure.subplots_adjust(left=.02, right=.98)\nplt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}